/**
 * 
 */
package de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.datacreation;
import java.io.BufferedReader;
import java.io.IOException;
/**
 * 
 * @author Christoph Teichmann
 * created Jun 2, 2014 9:27:03 PM
 * @version 0.1
 */
public interface Tokenizer
{
	/**
	 * 
	 *
	 * @param source
	 * @throws IOException
	 */
	void readFrom(BufferedReader source) throws IOException;
	/**
	 *
	 * @return
	 */
	boolean hasNext() throws IOException;
	/**
	 * 
	 *
	 * @return
	 */
	String currentWord();
	/**
	 * 
	 *
	 * @return
	 */
	String currentLanguage();
	/**
	 * 
	 */
	void setToNext() throws IOException;
}